# 1、 拿到网页源代码  urllib.request

# 2、 提取你想要的数据
#[{'rate': '1', 'movie': '肖申克的救赎', 'ratting_num': '9.7', 'person_num': '2339061'}, {'rate': '2', 'movie': '霸王别姬', 'ratting_num': '9.6', 'person_num': '1739772'}, {'rate': '3', 'movie': '阿甘正传', 'ratting_num': '9.5', 'person_num': '1761074'}, {'rate': '4', 'movie': '这个杀手不太冷', 'ratting_num': '9.4....


from urllib.request import Request, urlopen
import re


def get_page(url):
    req = Request(url, headers={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36"
    })  # 打开浏览器获取（谷歌是在检查-NetWork 第一条看你网页的链接，如https://movie.douban.com/top250   选择Headers，然后在Request Header）获取 User-Agent
    resp = urlopen(req)  # 发送请求
    return resp.read().decode("utf-8")  # 网页的charset是什么就是什么


# print(get_page("https://movie.douban.com/top250"))
# 下面都是正则对字符串的操作  我们想要<em>标签的排名和 <span>标签里的电影名字
# .*? 过滤掉两个标签之间的所有
# (.*?) 想要的内容那括号括起来
def parse_page(s):
    obj = re.compile(r'<div class="item">.*?<em class="">(?P<rate>.*?)</em>.*?'
                     r'<span class="title">(?P<movie>.*?)</span>.*?'
                     r'<span class="rating_num" property="v:average">(?P<ratting_num>.*?)'
                     r'</span>.*?<span>(?P<person_num>.*?)人评价</span>', re.S)  # re.S 可以让正则中的.匹配换行
    res = obj.finditer(s)
    lst = []
    for item in res:
        dic = item.groupdict()
        lst.append(dic)
    return lst


if __name__ == '__main__':
    with open("movie.txt",mode="w",encoding="utf-8") as f:
        for i in range(10):
            # s = get_page("https://movie.douban.com/top250")
            s = get_page(f"https://movie.douban.com/top250?start={i*25}&filter=")
            result = parse_page(s)
            for j in result:
                f.write(str(j))
                f.write("\n")
                print(j)
